package crawler.web.engine;

import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.InvocationTargetException;
import java.net.UnknownHostException;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;

import library.EnumLogType;
import library.Tracer;
import library.functions.Function1PV;
import library.thread.pool.ThreadPoolUtils;
import library.utils.file.FileUtils;
import crawler.web.beans.CrawlURL;
import crawler.web.engine.config.SpBN;
import crawler.web.engine.config.SpringUtils;
import crawler.web.engine.custom.TwoDBFrontier;
import crawler.web.framework.AnalyzerDedu;
import crawler.web.framework.AnalyzerPrs;
import crawler.web.framework.AnalyzerSim;
import crawler.web.framework.Crawler;
import crawler.web.framework.CrawlerContainer;

/**
 * multithread crawler
 * 
 * @author waynechen Feb 8, 2012 11:08:58 PM
 */
public class MultiThreadCrawl {

	/**
	 * 去掉main参数<br>
	 * 修改properies, 数据库名<br>
	 * 
	 * @author waynechen
	 * @param args
	 */
	public static void main(String[] args) {

		args = new String[] { "--reset", "failures" };
		args = new String[] { "--crawler", "sim" };
//		FileUtils.deleteDir(new File("source/crawler"));
		args = new String[] { "--list", "failures" };
		args = new String[] { "-h" };
		args = new String[] { "--list", "hosts" };
		args = new String[] { "--crawler", "prs" };
		args = new String[] { "--crawler", "dedu" };
		args = new String[] { "--crawler", "cc" };

		//转换参数形式
		List<String> argList = Arrays.asList(args);

		//查看库中信息
		if (argList.contains("--list")) {
			//查看hosts / 处理hosts到url queue
			if (argList.contains("hosts")) {
				printBDB(SpBN.bean_hosts_frontier);
				System.out.println("打印hosts");
			}
			//查看failure 队列 / 处理hosts到url queue
			if (argList.contains("failures")) {
				printBDB(SpBN.bean_failure_urls);
				System.out.println("打印失败url");
			}
			return;
		}

		//移动失败url到, 到等待队列
		if (argList.contains("--reset")) {
			if (argList.contains("failures")) {
				mvFailures();
				System.out.println("重置失败urls");
			}
			return;
		}

		//执行爬虫
		if (argList.contains("--crawler")) {
			processCrawler(argList);
			return;
		}

		//打印帮助
		processHelp();
	}

	private static void processCrawler(List<String> argList) {
		try {
			//日誌設置
			Tracer.setLogFilePath("source/crawl.log");
			Tracer.addTestingClass("crawler.*", "chen.*");
			Tracer.setLogType(EnumLogType.fileOnly);
			Tracer.setLogType(EnumLogType.consoleAndFile);
			Tracer.setLogType(EnumLogType.consoleOnly);
			//			Tracer.addTestingCatalog("info", "exception", "debug");
			//			Tracer.addTestingCatalog("info", "exception", "html", "url", "frequency");
			//			Tracer.addTestingCatalog("info", "exception", "html");
			//			Tracer.addTestingCatalog("data");
			Tracer.addTestingCatalog("info", "exception", "ratio");
			Tracer.startTracing();

			//			ThreadPoolUtils.initThreadPool(100);

			Integer time = SpringUtils.getBean(SpBN.conf_failure_stop_time);
			CrawlerContainer cc = SpringUtils.getBean("CrawlerContainer", CrawlerContainer.class);
			//初始化爬虫
			//并发执行爬虫
			while (true) {
				initCrawler(cc, argList);
				cc.collateralExecute(null);
				Thread.sleep(time * 1000 * 60);
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			Tracer.info("---crawler-done---");
			//flush 日志
			Tracer.stopTracing();
		}
	}

	private static void initCrawler(CrawlerContainer cc, List<String> list) throws NoSuchMethodException, InstantiationException, IllegalAccessException, InvocationTargetException, UnknownHostException, Exception {
		//添加10只虫子
		if (list.contains("cc")) {
			for (int i = 0; i < SpringUtils.getBeanInt(SpBN.conf_maxCrawler); i++)
				cc.addSubNode(SpringUtils.getBean("Crawler", Crawler.class));
		}

		//设置最大爬虫数
		cc.setMaxSubNodeNumber(SpringUtils.getBeanInt(SpBN.conf_maxCrawler));

		//加入分析器AnalyzerPrs
		if (list.contains("prs")) {
			for (int i = 0; i < 1; i++)
				cc.addSubNode(SpringUtils.getBean("AnalyzerPrs", AnalyzerPrs.class));
		}

		//加入分析器AnalyzerSim
		if (list.contains("sim")) {
			for (int i = 0; i < 1; i++)
				cc.addSubNode(SpringUtils.getBean("AnalyzerSim", AnalyzerSim.class));
		}

		//加入分析器AnalyzerDedu
		if (list.contains("dedu")) {
			for (int i = 0; i < 1; i++)
				cc.addSubNode(SpringUtils.getBean("AnalyzerDedu", AnalyzerDedu.class));
		}
	}

	private static void mvFailures() {
		BDBFrontier failures = SpringUtils.getBean(SpBN.bean_failure_urls);
		final TwoDBFrontier urls = SpringUtils.getBean(SpBN.bean_url_frontier);
		failures.dump(new Function1PV<Map.Entry<String, CrawlURL>>() {
			@Override
			public void apply(Entry<String, CrawlURL> a) {
				try {
					a.getValue().setRetryTimes(0);
					urls.putBack(a.getValue());
				} catch (Exception e) {
					e.printStackTrace();
				}
			}
		});
	}

	private static void printBDB(String beanName) {
		BDBFrontier hosts = SpringUtils.getBean(beanName);
		hosts.dump(new Function1PV<Map.Entry<String, CrawlURL>>() {
			@Override
			public void apply(Entry<String, CrawlURL> a) {
				System.out.println(a.getKey());
			}
		});
	}

	private static void processHelp() {
		BufferedReader reader = new BufferedReader(new InputStreamReader(ClassLoader.getSystemResourceAsStream("help")));
		try {
			for (String line; (line = reader.readLine()) != null;) {
				System.out.println(line);
			}
		} catch (IOException e) {
			e.printStackTrace();
		}
		try {
			reader.close();
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}

//
//
//
//
//
//
//